#!/usr/bin/env python3
"""
域名统计脚本
分析清洗后的接口数据，按域名进行统计分析
"""

import json
import re
from urllib.parse import urlparse
from collections import defaultdict, Counter
from datetime import datetime
import os

class DomainStatistics:
    """域名统计分析器"""

    def __init__(self, cleaned_data_file):
        """
        初始化域名统计器

        Args:
            cleaned_data_file: 清洗后的JSON文件路径
        """
        self.cleaned_data_file = cleaned_data_file
        self.data = None
        self.interfaces = []

    def load_data(self):
        """加载清洗后的数据"""
        try:
            with open(self.cleaned_data_file, 'r', encoding='utf-8') as f:
                self.data = json.load(f)

            self.interfaces = self.data.get('interfaces', [])
            print(f"✅ 成功加载数据文件: {self.cleaned_data_file}")
            print(f"📊 总接口数: {len(self.interfaces)}")
            return True
        except Exception as e:
            print(f"❌ 加载数据失败: {e}")
            return False

    def extract_domain(self, url):
        """
        从URL中提取域名

        Args:
            url: URL字符串

        Returns:
            str: 域名，如果无法提取则返回'unknown'
        """
        if not url:
            return 'unknown'

        try:
            # 如果是完整URL，解析域名
            if url.startswith(('http://', 'https://')):
                parsed = urlparse(url)
                return parsed.netloc.lower()
            else:
                # 相对路径，归类为'local'或'unknown'
                return 'local_path'
        except Exception:
            return 'unknown'

    def categorize_domain(self, domain):
        """
        对域名进行分类

        Args:
            domain: 域名

        Returns:
            str: 域名分类
        """
        if domain == 'local_path':
            return '本地路径'
        elif domain == 'unknown':
            return '未知域名'

        # 常见域名分类
        tech_companies = {
            'googleapis.com': 'Google服务',
            'google.com': 'Google服务',
            'youtube.com': 'Google服务',
            'gstatic.com': 'Google服务',
            'microsoft.com': 'Microsoft服务',
            'github.com': '开发工具',
            'api.github.com': '开发工具',
            'aws.amazon.com': 'AWS服务',
            'amazonaws.com': 'AWS服务',
            'facebook.com': '社交媒体',
            'twitter.com': '社交媒体',
            'instagram.com': '社交媒体',
            'linkedin.com': '社交媒体',
            'baidu.com': '百度服务',
            'qq.com': '腾讯服务',
            'weibo.com': '社交媒体',
            'taobao.com': '电商服务',
            'tmall.com': '电商服务',
            'jd.com': '电商服务'
        }

        # 检查是否匹配已知域名
        for known_domain, category in tech_companies.items():
            if known_domain in domain:
                return category

        # 根据域名模式分类
        if 'api.' in domain:
            return 'API服务'
        elif '.cdn.' in domain or 'cdn' in domain:
            return 'CDN服务'
        elif 'analytics' in domain or 'tracking' in domain:
            return '分析追踪'
        elif 'ad.' in domain or 'ads' in domain:
            return '广告服务'
        elif 'static' in domain or 'assets' in domain:
            return '静态资源'
        elif domain.endswith('.gov.cn') or domain.endswith('.gov'):
            return '政府网站'
        elif domain.endswith('.edu.cn') or domain.endswith('.edu'):
            return '教育网站'
        elif domain.endswith('.com'):
            return '商业网站'
        elif domain.endswith('.cn'):
            return '中国网站'
        elif domain.endswith('.org'):
            return '组织网站'
        elif domain.endswith('.net'):
            return '网络服务'
        else:
            return '其他域名'

    def analyze_domains(self):
        """
        分析域名统计信息

        Returns:
            dict: 域名统计结果
        """
        if not self.interfaces:
            print("❌ 没有接口数据可供分析")
            return None

        print("🔍 开始分析域名统计...")

        # 统计数据结构
        domain_stats = defaultdict(lambda: {
            'count': 0,
            'methods': Counter(),
            'status_codes': Counter(),
            'urls': set(),
            'category': '',
            'avg_response_size': 0,
            'total_response_size': 0
        })

        total_size = 0
        total_count = 0

        for interface in self.interfaces:
            url = interface.get('url', '')
            method = interface.get('method', 'GET')
            status_code = interface.get('response', {}).get('status_code', 0)
            response_size = interface.get('response', {}).get('size', 0)

            # 提取域名
            domain = self.extract_domain(url)

            # 分类域名
            category = self.categorize_domain(domain)

            # 更新统计
            domain_stats[domain]['count'] += 1
            domain_stats[domain]['methods'][method] += 1
            domain_stats[domain]['status_codes'][status_code] += 1
            domain_stats[domain]['urls'].add(url)
            domain_stats[domain]['category'] = category
            domain_stats[domain]['total_response_size'] += response_size

            total_size += response_size
            total_count += 1

        # 计算平均响应大小
        for domain, stats in domain_stats.items():
            if stats['count'] > 0:
                stats['avg_response_size'] = stats['total_response_size'] / stats['count']
            # 转换set为list以便JSON序列化
            stats['urls'] = list(stats['urls'])
            # 转换Counter为dict
            stats['methods'] = dict(stats['methods'])
            stats['status_codes'] = dict(stats['status_codes'])

        print(f"✅ 域名分析完成！")
        print(f"📊 发现 {len(domain_stats)} 个不同域名")

        return dict(domain_stats)

    def generate_category_stats(self, domain_stats):
        """
        生成分类统计

        Args:
            domain_stats: 域名统计结果

        Returns:
            dict: 分类统计结果
        """
        category_stats = defaultdict(lambda: {
            'count': 0,
            'domains': set(),
            'total_interfaces': 0
        })

        for domain, stats in domain_stats.items():
            category = stats['category']
            count = stats['count']

            category_stats[category]['count'] += 1
            category_stats[category]['domains'].add(domain)
            category_stats[category]['total_interfaces'] += count

        # 转换set为list
        for category, stats in category_stats.items():
            stats['domains'] = list(stats['domains'])

        return dict(category_stats)

    def generate_report(self, domain_stats, category_stats):
        """
        生成统计报告

        Args:
            domain_stats: 域名统计结果
            category_stats: 分类统计结果

        Returns:
            dict: 完整的统计报告
        """
        # 排序域名（按接口数量降序）
        sorted_domains = sorted(
            domain_stats.items(),
            key=lambda x: x[1]['count'],
            reverse=True
        )

        # 排序分类（按接口数量降序）
        sorted_categories = sorted(
            category_stats.items(),
            key=lambda x: x[1]['total_interfaces'],
            reverse=True
        )

        # 计算总体统计
        total_interfaces = sum(stats['count'] for stats in domain_stats.values())
        total_domains = len(domain_stats)
        top_domain = sorted_domains[0] if sorted_domains else ('unknown', {'count': 0})

        report = {
            "metadata": {
                "generated_at": datetime.now().isoformat(),
                "source_file": self.cleaned_data_file,
                "total_interfaces": total_interfaces,
                "total_domains": total_domains,
                "top_domain": top_domain[0],
                "top_domain_count": top_domain[1]['count']
            },
            "summary": {
                "domains": total_domains,
                "interfaces": total_interfaces,
                "avg_interfaces_per_domain": total_interfaces / total_domains if total_domains > 0 else 0
            },
            "domain_rankings": {
                "by_interface_count": [
                    {
                        "rank": i + 1,
                        "domain": domain,
                        "category": stats['category'],
                        "interface_count": stats['count'],
                        "methods": stats['methods'],
                        "status_codes": stats['status_codes'],
                        "avg_response_size": round(stats['avg_response_size'], 2),
                        "sample_urls": stats['urls'][:5]  # 只显示前5个URL示例
                    }
                    for i, (domain, stats) in enumerate(sorted_domains[:20])  # 只显示前20个
                ]
            },
            "category_rankings": {
                "by_interface_count": [
                    {
                        "rank": i + 1,
                        "category": category,
                        "domain_count": stats['count'],
                        "total_interfaces": stats['total_interfaces'],
                        "domains": stats['domains'][:10]  # 只显示前10个域名
                    }
                    for i, (category, stats) in enumerate(sorted_categories)
                ]
            },
            "detailed_stats": domain_stats
        }

        return report

    def save_report(self, report, output_file=None):
        """
        保存统计报告

        Args:
            report: 统计报告
            output_file: 输出文件路径
        """
        if output_file is None:
            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            output_file = f"数据清洗/domain_statistics_{timestamp}.json"

        try:
            with open(output_file, 'w', encoding='utf-8') as f:
                json.dump(report, f, indent=2, ensure_ascii=False)
            print(f"✅ 报告已保存: {output_file}")
            return output_file
        except Exception as e:
            print(f"❌ 保存报告失败: {e}")
            return None

    def filter_interfaces_by_domain(self, target_domains, output_file=None):
        """
        按域名过滤接口数据

        Args:
            target_domains: 目标域名列表
            output_file: 输出文件路径

        Returns:
            str: 过滤后的数据文件路径
        """
        if not self.interfaces:
            print("❌ 没有接口数据可供过滤")
            return None

        print(f"🔍 开始按域名过滤接口...")
        print(f"🎯 目标域名: {target_domains}")

        # 标准化目标域名（转为小写）
        target_domains = [domain.lower() for domain in target_domains]

        filtered_interfaces = []
        stats = defaultdict(int)

        for interface in self.interfaces:
            url = interface.get('url', '')
            domain = self.extract_domain(url).lower()

            # 检查是否匹配目标域名
            if domain in target_domains:
                filtered_interfaces.append(interface)
                stats[domain] += 1

        if not filtered_interfaces:
            print(f"❌ 没有找到匹配域名的接口")
            return None

        print(f"✅ 过滤完成！")
        print(f"📊 过滤结果:")
        for domain, count in stats.items():
            print(f"  - {domain}: {count} 个接口")

        # 生成过滤后的数据
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        if output_file is None:
            domain_suffix = "_".join([d.replace(".", "_") for d in target_domains[:3]])  # 最多显示3个域名
            output_file = f"数据清洗/filtered_interfaces_{domain_suffix}_{timestamp}.json"

        filtered_data = {
            "metadata": {
                "filtered_at": datetime.now().isoformat(),
                "source_file": self.cleaned_data_file,
                "target_domains": target_domains,
                "total_filtered_interfaces": len(filtered_interfaces),
                "domain_counts": dict(stats)
            },
            "interfaces": filtered_interfaces,
            "summary": {
                "total_domains": len(stats),
                "total_interfaces": len(filtered_interfaces),
                "avg_interfaces_per_domain": len(filtered_interfaces) / len(stats)
            }
        }

        try:
            with open(output_file, 'w', encoding='utf-8') as f:
                json.dump(filtered_data, f, indent=2, ensure_ascii=False)
            print(f"✅ 过滤后的数据已保存: {output_file}")
            return output_file
        except Exception as e:
            print(f"❌ 保存过滤数据失败: {e}")
            return None

    def print_summary(self, report):
        """
        打印统计摘要

        Args:
            report: 统计报告
        """
        metadata = report['metadata']
        summary = report['summary']
        domain_rankings = report['domain_rankings']['by_interface_count']
        category_rankings = report['category_rankings']['by_interface_count']

        print("\n" + "="*60)
        print("📊 域名统计报告")
        print("="*60)

        print(f"\n📋 总体概况:")
        print(f"  总接口数: {metadata['total_interfaces']:,}")
        print(f"  总域名数: {metadata['total_domains']}")
        print(f"  平均每域名接口数: {summary['avg_interfaces_per_domain']:.1f}")
        print(f"  最热门域名: {metadata['top_domain']} ({metadata['top_domain_count']} 个接口)")

        print(f"\n🏆 Top 10 域名:")
        for i, domain_info in enumerate(domain_rankings[:10]):
            domain = domain_info['domain']
            count = domain_info['interface_count']
            category = domain_info['category']
            print(f"  {i+1:2d}. {domain:<30} ({count:4d} 接口) - {category}")

        print(f"\n📂 分类统计:")
        for i, category_info in enumerate(category_rankings[:8]):
            category = category_info['category']
            total = category_info['total_interfaces']
            domain_count = category_info['domain_count']
            print(f"  {i+1}. {category:<15} ({total:4d} 接口, {domain_count} 域名)")

        print(f"\n⏰ 生成时间: {metadata['generated_at']}")
        print("="*60)

def interactive_domain_filter(analyzer):
    """
    交互式域名过滤

    Args:
        analyzer: DomainStatistics实例
    """
    print("\n" + "="*50)
    print("🔍 域名过滤功能")
    print("="*50)

    # 显示所有可用域名
    if not analyzer.interfaces:
        print("❌ 没有接口数据可供过滤")
        return

    # 统计所有域名
    domain_counts = defaultdict(int)
    for interface in analyzer.interfaces:
        url = interface.get('url', '')
        domain = analyzer.extract_domain(url)
        domain_counts[domain] += 1

    print(f"\n📋 发现 {len(domain_counts)} 个不同域名:")

    # 按数量排序显示
    sorted_domains = sorted(domain_counts.items(), key=lambda x: x[1], reverse=True)

    # 显示编号列表
    print("编号  域名                                    接口数量")
    print("-" * 70)
    for i, (domain, count) in enumerate(sorted_domains, 1):
        print(f"{i:3d}.  {domain:<40} ({count:4d} 接口)")

    print("\n🎯 过滤选项:")
    print("1. 按编号选择域名")
    print("2. 输入域名关键词")
    print("3. 按分类过滤")
    print("4. 选择Top N域名")
    print("5. 返回主菜单")

    choice = input("\n请选择过滤方式 (1-5): ").strip()

    target_domains = []

    if choice == "1":
        # 按编号选择
        try:
            numbers = input("请输入域名编号 (用逗号分隔，如: 1,3,5): ").strip()
            selected_indices = [int(n.strip()) for n in numbers.split(",") if n.strip().isdigit()]

            for idx in selected_indices:
                if 1 <= idx <= len(sorted_domains):
                    target_domains.append(sorted_domains[idx-1][0])

            if target_domains:
                print(f"✅ 选择了 {len(target_domains)} 个域名")

        except ValueError:
            print("❌ 输入格式错误")

    elif choice == "2":
        # 关键词搜索
        keyword = input("请输入域名关键词: ").strip().lower()
        if keyword:
            for domain, count in sorted_domains:
                if keyword in domain.lower():
                    target_domains.append(domain)

            if target_domains:
                print(f"✅ 找到 {len(target_domains)} 个匹配域名")
            else:
                print(f"❌ 没有找到包含 '{keyword}' 的域名")

    elif choice == "3":
        # 按分类过滤
        print("\n可用分类:")
        categories = set()
        for domain, count in sorted_domains:
            category = analyzer.categorize_domain(domain)
            categories.add(category)

        for i, category in enumerate(sorted(categories), 1):
            print(f"{i}. {category}")

        try:
            cat_choice = int(input(f"请选择分类 (1-{len(categories)}): ").strip())
            selected_category = sorted(categories)[cat_choice-1]

            for domain, count in sorted_domains:
                if analyzer.categorize_domain(domain) == selected_category:
                    target_domains.append(domain)

            print(f"✅ 选择了分类 '{selected_category}' 的 {len(target_domains)} 个域名")

        except (ValueError, IndexError):
            print("❌ 选择无效")

    elif choice == "4":
        # 选择Top N
        try:
            n = int(input("请输入要选择的Top N数量: ").strip())
            if n > 0:
                target_domains = [domain for domain, count in sorted_domains[:n]]
                print(f"✅ 选择了Top {n} 个域名")
            else:
                print("❌ 数量必须大于0")
        except ValueError:
            print("❌ 请输入有效数字")

    elif choice == "5":
        return

    else:
        print("❌ 无效选择")
        return

    # 执行过滤
    if target_domains:
        print(f"\n🎯 目标域名: {target_domains}")
        confirm = input("确认执行过滤? (y/n): ").strip().lower()

        if confirm == 'y':
            output_file = analyzer.filter_interfaces_by_domain(target_domains)
            if output_file:
                print(f"✅ 过滤完成! 结果保存在: {output_file}")

                # 询问是否继续过滤
                continue_filter = input("\n是否继续按其他条件过滤? (y/n): ").strip().lower()
                if continue_filter == 'y':
                    interactive_domain_filter(analyzer)
            else:
                print("❌ 过滤失败")
        else:
            print("❌ 已取消过滤")
    else:
        print("❌ 没有选择有效的域名")

def main():
    """主函数"""
    # 查找最新的清洗数据文件
    data_dir = "数据清洗"
    if not os.path.exists(data_dir):
        print(f"❌ 目录不存在: {data_dir}")
        return

    # 查找清洗后的数据文件
    cleaned_files = [f for f in os.listdir(data_dir) if f.startswith('fixed_cleaned_') and f.endswith('.json')]
    if not cleaned_files:
        print(f"❌ 在 {data_dir} 目录中没有找到清洗后的数据文件")
        print("请先运行数据清洗脚本生成数据文件")
        return

    # 选择最新的文件
    cleaned_files.sort()
    latest_file = os.path.join(data_dir, cleaned_files[-1])

    print(f"🎯 使用数据文件: {latest_file}")

    # 创建域名统计器
    analyzer = DomainStatistics(latest_file)

    # 加载数据
    if not analyzer.load_data():
        return

    while True:
        print("\n" + "="*50)
        print("📊 域名统计分析工具")
        print("="*50)
        print("1. 生成完整域名统计报告")
        print("2. 按域名过滤接口 (二次清洗)")
        print("3. 生成可视化图表")
        print("4. 退出")

        choice = input("\n请选择功能 (1-4): ").strip()

        if choice == "1":
            # 生成完整统计
            print("\n🔍 开始分析域名统计...")

            # 分析域名
            domain_stats = analyzer.analyze_domains()
            if not domain_stats:
                continue

            # 生成分类统计
            category_stats = analyzer.generate_category_stats(domain_stats)

            # 生成报告
            report = analyzer.generate_report(domain_stats, category_stats)

            # 保存报告
            output_file = analyzer.save_report(report)

            # 打印摘要
            analyzer.print_summary(report)

            if output_file:
                print(f"\n📄 详细报告已保存到: {output_file}")
                print("可以查看JSON文件获取更详细的统计信息")

        elif choice == "2":
            # 域名过滤
            interactive_domain_filter(analyzer)

        elif choice == "3":
            # 可视化
            try:
                from 域名统计可视化 import DomainVisualizer

                # 查找统计文件
                stats_files = [f for f in os.listdir(data_dir) if f.startswith('domain_statistics_') and f.endswith('.json')]
                if not stats_files:
                    print("❌ 没有找到域名统计文件，请先生成统计报告")
                    continue

                stats_files.sort()
                latest_stats = os.path.join(data_dir, stats_files[-1])

                print(f"🎨 使用统计文件: {latest_stats}")
                visualizer = DomainVisualizer(latest_stats)
                charts = visualizer.generate_all_charts()

                if charts:
                    print(f"✅ 可视化完成！生成了 {len([c for c in charts.values() if c])} 个图表")
                    for chart_type, chart_path in charts.items():
                        if chart_path:
                            print(f"  - {chart_type}: {chart_path}")
                else:
                    print("❌ 没有成功生成任何图表")

            except ImportError:
                print("❌ 无法导入可视化模块，请确保域名统计可视化.py文件存在")
            except Exception as e:
                print(f"❌ 生成可视化时出错: {e}")

        elif choice == "4":
            print("👋 退出程序")
            break

        else:
            print("❌ 无效选择，请重新输入")

if __name__ == "__main__":
    main()